package cn.xiaolang.function.blog;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;


public class CSDNBlogApplicationDemo {


    public static void main(String[] args) throws InterruptedException {


        List<String> list = new ArrayList<>();
        list.add("https://blog.csdn.net/u012031755/article/details/117230769");
        list.add("https://blog.csdn.net/u012031755/article/details/113992663");
        list.add("https://blog.csdn.net/u012031755/article/details/80806438");
        list.add("https://blog.csdn.net/u012031755/article/details/115917640");

        for (int i = 0; i <= 1000; i++) {
            // 40000-70000
            Double waitingTime = Math.random() * 40000 + 30000;
            Thread.sleep(waitingTime.longValue());
            System.out.println("第[{" + i + "}]次抓取");
            for (int j = 0; j < list.size(); j++) {
                String url = list.get(j);
                Document doc = getDocument(url);
                if (null != doc) {
                    System.out.println(i + "-url: " + url);
                }
            }
        }
    }


    /**
     * Jsoup打开连接地址获取Document对象
     *
     * @param url
     * @return
     */
    public static Document getDocument(String url) {
        try {
            Connection conn = Jsoup.connect(url).ignoreContentType(true).ignoreHttpErrors(true).userAgent("Mozilla");
            Document document = conn.timeout(2000).get();
            if (null != document) {// 表示ip被拦截或者其他情况
                return document;
            }
        } catch (Exception e) {
            System.out.println("抓取失败..." + e.getMessage());
        }
        return null;
    }

}